{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "已获取0篇文章: 0it [00:00, ?it/s]\u001b[A\n",
      "已获取0篇文章: 3it [00:00, 22.70it/s]\u001b[A\n",
      "已获取0篇文章: 5it [00:00, 16.16it/s]\u001b[A\n",
      "已获取0篇文章: 8it [00:00, 16.47it/s]\u001b[A\n",
      "已获取0篇文章: 14it [00:00, 23.68it/s]\u001b[A\n",
      "已获取0篇文章: 19it [00:00, 25.94it/s]\u001b[A\n",
      "已获取0篇文章: 22it [00:01, 17.78it/s]\u001b[A\n",
      "已获取0篇文章: 25it [00:01, 17.94it/s]\u001b[A\n",
      "已获取0篇文章: 28it [00:01, 16.59it/s]\u001b[A\n",
      "已获取0篇文章: 30it [00:01, 16.10it/s]\u001b[A\n",
      "已获取0篇文章: 34it [00:02, 16.43it/s]\u001b[A\n",
      "已获取0篇文章: 36it [00:02, 16.32it/s]\u001b[A\n",
      "已获取0篇文章: 46it [00:02, 18.97it/s]\u001b[A\n",
      "已获取0篇文章: 51it [00:02, 19.98it/s]\u001b[A\n",
      "已获取0篇文章: 55it [00:02, 20.36it/s]\u001b[A\n",
      "已获取0篇文章: 58it [00:03, 18.21it/s]\u001b[A\n",
      "已获取0篇文章: 61it [00:03, 18.33it/s]\u001b[A\n",
      "已获取0篇文章: 68it [00:03, 19.64it/s]\u001b[A\n",
      "已获取0篇文章: 73it [00:03, 20.44it/s]\u001b[A\n",
      "已获取0篇文章: 80it [00:03, 20.50it/s]\u001b[A\n",
      "已获取0篇文章: 84it [00:04, 19.90it/s]Exception in thread Thread-4:\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/fwj/anaconda3/lib/python3.6/threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"/home/fwj/anaconda3/lib/python3.6/site-packages/tqdm/_tqdm.py\", line 144, in run\n",
      "    for instance in self.tqdm_cls._instances:\n",
      "  File \"/home/fwj/anaconda3/lib/python3.6/_weakrefset.py\", line 60, in __iter__\n",
      "    for itemref in self.data:\n",
      "RuntimeError: Set changed size during iteration\n",
      "\n",
      "已获取996200篇文章: : 3096184it [2:10:10, 396.42it/s]\n"
     ]
    }
   ],
   "source": [
    "from gensim.corpora.wikicorpus import extract_pages,filter_wiki\n",
    "import bz2file\n",
    "import re\n",
    "from tqdm import tqdm\n",
    "import codecs\n",
    "\n",
    "wiki = extract_pages(bz2file.open('zhwiki-20180301-pages-articles-multistream.xml.bz2'))\n",
    "from opencc import OpenCC \n",
    "\n",
    "openCC = OpenCC('hk2s')  # convert from Simplified Chinese to Traditional Chinese\n",
    "# can also set conversion by calling set_conversion\n",
    "# openCC.set_conversion('s2tw')\n",
    "to_convert = '开放中文转换'\n",
    "converted = openCC.convert(to_convert)\n",
    "\n",
    "def wiki_replace(d):\n",
    "    \n",
    "    s = d[1]\n",
    "    s = re.sub(':*{\\|[\\s\\S]*?\\|}', '', s)\n",
    "    s = re.sub('<gallery>[\\s\\S]*?</gallery>', '', s)\n",
    "    s = re.sub('(.){{([^{}\\n]*?\\|[^{}\\n]*?)}}', '\\\\1[[\\\\2]]', s)\n",
    "    s = filter_wiki(s)\n",
    "    s = re.sub('\\* *\\n|\\'{2,}', '', s)\n",
    "    s = re.sub('\\n+', '\\n', s)\n",
    "    s = re.sub('\\n[:;]|\\n +', '\\n', s)\n",
    "    s = re.sub('\\n==', '\\n\\n==', s)\n",
    "    s = u'【' + d[0] + u'】\\n' + s\n",
    "    return openCC.convert(s).strip()\n",
    "\n",
    "i = 0\n",
    "f = codecs.open('wiki.txt', 'w', encoding='utf-8')\n",
    "w = tqdm(wiki, desc=u'已获取0篇文章')\n",
    "for d in w:\n",
    "    if not re.findall('^[a-zA-Z]+:', d[0]) and d[0] and not re.findall(u'^#', d[1]):\n",
    "        s = wiki_replace(d)\n",
    "        f.write(s+'\\n\\n\\n')\n",
    "        i += 1\n",
    "        if i % 100 == 0:\n",
    "            w.set_description(u'已获取%s篇文章'%i)\n",
    "\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
